package util;

import java.io.File;
import java.io.IOException;
import java.net.URL;

import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import websphinx.Crawler;
import websphinx.DownloadParameters;
import websphinx.Page;

public class SimpleCrawler extends Crawler {

	private static final long serialVersionUID = 1000L;

	protected final Log log = LogFactory.getLog(getClass());

	private String targetDir;

	public SimpleCrawler() {
		super();

		DownloadParameters dp = new DownloadParameters();
		dp.changeObeyRobotExclusion(true);
		setDownloadParameters(dp);
		setDomain(Crawler.SUBTREE);
		setLinkType(Crawler.HYPERLINKS);
		setMaxDepth(3);
	}

	public void setTargetDir(String targetDir) {
		this.targetDir = targetDir;
	}

	public void init() throws Exception {
		File targetDirFile = new File(targetDir);
		if (targetDirFile.exists()) {
			FileUtils.forceDelete(targetDirFile);
		}
		FileUtils.forceMkdir(targetDirFile);
	}

	@Override
	public void visit(Page page) {
		URL url = page.getURL();

		String fixed = url.toString();
		fixed = fixed.replace("://", "_"); // for the http:// part
		fixed = fixed.replace(".", "_"); // for the www.engr.sjsu.edu part
		fixed = fixed.replace("/", "#%"); // for the all ges/blah/blah part

		File local = new File("html/" + fixed);
		
		try {
			if (!url.toString().contains("pdf")) { // can't work with PDFs
				FileUtils.copyURLToFile(url, local);
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		try {
			Thread.sleep(1000L);
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
	}
}
